{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bIKqmN_5jpJB"
   },
   "source": [
    "# Quickstart example\n",
    "\n",
    "This notebook shows you a quick way to get started with [pypdf_table_extraction](https://github.com/py-pdf/pypdf_table_extraction) .\n",
    "\n",
    "**Usage:** Either upload files or provide a PDF URL in the specified cells."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "9xfD4-FXfgFO"
   },
   "outputs": [],
   "source": [
    "# @title 🛠️ Install [pypdf_table_extraction](https://github.com/py-pdf/pypdf_table_extraction)\n",
    "!pip install pypdf-table-extraction\n",
    "# install tabulate (optional) only needed in this notebook for pretty display of results.\n",
    "!pip install tabulate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Bootstrap and common imports\n",
    "import os, sys, time\n",
    "sys.path.insert(0, os.path.abspath('')) # Prefer the local version if available\n",
    "\n",
    "import pypdf_table_extraction\n",
    "\n",
    "print(f\"Using pypdf_table_extraction v{pypdf_table_extraction.__version__} from file {pypdf_table_extraction.__file__}.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "form",
    "id": "x2gOsT54QO6f"
   },
   "outputs": [],
   "source": [
    "# @title 📂 Create necessary directories and delete `sample_data` if exists\n",
    "\n",
    "import os\n",
    "import shutil\n",
    "from pathlib import Path\n",
    "\n",
    "# Function to delete a directory and its contents\n",
    "def delete_directory(path):\n",
    "    try:\n",
    "        shutil.rmtree(path)\n",
    "        print(f\"Deleted directory: {path}\")\n",
    "    except FileNotFoundError:\n",
    "        print(f\"Directory not found: {path}\")\n",
    "    except Exception as e:\n",
    "        print(f\"Error deleting directory {path}: {e}\")\n",
    "\n",
    "# Delete /content/sample_data if it exists\n",
    "sample_data_dir = Path('/content/sample_data')\n",
    "if sample_data_dir.exists():\n",
    "    print(\"Deleting /content/sample_data directory...\")\n",
    "    delete_directory(sample_data_dir)\n",
    "\n",
    "# Create the necessary directories\n",
    "os.makedirs('/content/output', exist_ok=True)\n",
    "os.makedirs('/content/sample_pdfs', exist_ok=True)\n",
    "\n",
    "# Define input and output directories\n",
    "input_dir = Path('/content/sample_pdfs')\n",
    "output_dir = Path('/content/output')\n",
    "\n",
    "print(\"Directories set up complete.\")\n",
    "print(f\"Input directory: {input_dir}\")\n",
    "print(f\"Output directory: {output_dir}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "form",
    "id": "l3kChPYC8Srj"
   },
   "outputs": [],
   "source": [
    "# @title 📤 Upload Files (Optional)\n",
    "\n",
    "from google.colab import files\n",
    "\n",
    "print(\"\\nPlease upload your PDF files. They will be saved in /content/sample_pdfs\")\n",
    "\n",
    "# Upload files\n",
    "uploaded = files.upload()\n",
    "\n",
    "# Move uploaded files to /content/sample_pdfs and remove from /content\n",
    "for filename in uploaded.keys():\n",
    "    src_path = Path('/content') / filename\n",
    "    dst_path = Path('/content/sample_pdfs') / filename\n",
    "    if src_path.exists():\n",
    "        shutil.move(str(src_path), str(dst_path))\n",
    "        print(f\"Moved {filename} to /content/sample_pdfs\")\n",
    "    else:\n",
    "        print(f\"Warning: {filename} not found in /content\")\n",
    "\n",
    "# Clean up any remaining PDF files in /content\n",
    "for file in Path('/content').glob('*.pdf'):\n",
    "    os.remove(file)\n",
    "    print(f\"Removed {file.name} from /content\")\n",
    "\n",
    "print(\"\\nUpload and organization complete. Files are now only in /content/sample_pdfs\")\n",
    "\n",
    "# Verify contents of /content/sample_pdfs\n",
    "print(\"\\nContents of /content/sample_pdfs:\")\n",
    "print(os.listdir('/content/sample_pdfs'))\n",
    "\n",
    "# Verify no PDF files in /content\n",
    "print(\"\\nChecking for PDF files in /content:\")\n",
    "content_pdfs = list(Path('/content').glob('*.pdf'))\n",
    "if content_pdfs:\n",
    "    print(\"Warning: Found these PDF files in /content:\")\n",
    "    for pdf in content_pdfs:\n",
    "        print(f\" - {pdf.name}\")\n",
    "else:\n",
    "    print(\"No PDF files found in /content\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "form",
    "id": "rdNIDYtlg5ME"
   },
   "outputs": [],
   "source": [
    "# @title ⬇📕 Download Sample .PDF Document (Optional)\n",
    "\n",
    "# import os\n",
    "import requests\n",
    "# from pathlib import Path\n",
    "\n",
    "def convert_github_url_to_raw(url):\n",
    "    if \"github.com\" in url and \"/blob/\" in url:\n",
    "        raw_url = url.replace(\"github.com\", \"raw.githubusercontent.com\").replace(\"/blob/\", \"/\")\n",
    "        return raw_url\n",
    "    else:\n",
    "        return \"Invalid GitHub URL\"\n",
    "\n",
    "# Sample .pdf data from GitHub\n",
    "pdf_url = 'https://github.com/py-pdf/pypdf_table_extraction/blob/main/docs/_static/pdf/foo.pdf' # @param {type:\"string\"}\n",
    "\n",
    "# Convert the GitHub URL to the raw content URL\n",
    "pdf_url = convert_github_url_to_raw(pdf_url)\n",
    "\n",
    "# Check if the URL is valid\n",
    "if pdf_url == \"Invalid GitHub URL\":\n",
    "    raise ValueError(\"The provided GitHub URL is invalid.\")\n",
    "\n",
    "# Create the /content/sample_pdfs directory if it doesn't exist\n",
    "sample_pdfs_dir = Path('/content/sample_pdfs')\n",
    "sample_pdfs_dir.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "# Download the PDF\n",
    "response = requests.get(pdf_url)\n",
    "response.raise_for_status()  # Check if the request was successful\n",
    "\n",
    "# Extract the filename from the URL\n",
    "filename = os.path.basename(pdf_url)\n",
    "\n",
    "# Specify the file path in the /content/sample_pdfs directory\n",
    "pdf_file_path = sample_pdfs_dir / filename\n",
    "\n",
    "# Save the file, overwriting if it already exists\n",
    "with open(pdf_file_path, 'wb') as file:\n",
    "    file.write(response.content)\n",
    "\n",
    "print(f\"PDF file downloaded and saved to: {pdf_file_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "K24xxrxtXDgf"
   },
   "outputs": [],
   "source": [
    "# import pypdf_table_extraction\n",
    "import logging\n",
    "import pandas as pd\n",
    "# from pathlib import Path\n",
    "from pypdf import PdfReader\n",
    "from IPython.display import display\n",
    "\n",
    "# Set up logging\n",
    "logging.getLogger(\"camelot\").setLevel(logging.DEBUG)\n",
    "logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')\n",
    "\n",
    "def process_pdf(pdf_file, output_dir):\n",
    "    print(f\"Processing {pdf_file.name}\")\n",
    "    logging.info(f\"Processing {pdf_file.name}\")\n",
    "\n",
    "    # Verify PDF can be opened with PdfReader before processing\n",
    "    try:\n",
    "        reader = PdfReader(str(pdf_file))\n",
    "        if len(reader.pages) == 0:\n",
    "            raise ValueError(f\"No pages found in PDF {pdf_file.name}\")\n",
    "    except Exception as e:\n",
    "        print(f\"Failed to open PDF {pdf_file.name} with PdfReader: {e}\")\n",
    "        logging.error(f\"Failed to open PDF {pdf_file.name} with PdfReader: {e}\")\n",
    "        return\n",
    "\n",
    "    # Read tables from the PDF using pypdf_table_extraction\n",
    "    try:\n",
    "        tables = pypdf_table_extraction.read_pdf(str(pdf_file))\n",
    "    except Exception as e:\n",
    "        print(f\"Failed to read PDF {pdf_file.name}: {e}\")\n",
    "        logging.error(f\"Failed to read PDF {pdf_file.name}: {e}\")\n",
    "        return\n",
    "\n",
    "    if len(tables) == 0:\n",
    "        print(f\"No tables detected in {pdf_file.name}\")\n",
    "        logging.warning(f\"No tables detected in {pdf_file.name}\")\n",
    "        return\n",
    "\n",
    "    # Create a subdirectory for this PDF's output\n",
    "    pdf_output_dir = output_dir / pdf_file.stem\n",
    "    pdf_output_dir.mkdir(exist_ok=True)\n",
    "\n",
    "    # Process individual tables\n",
    "    for i, table in enumerate(tables):\n",
    "        try:\n",
    "            # Convert table to pandas DataFrame\n",
    "            df = table.df\n",
    "\n",
    "            # Display the DataFrame\n",
    "            print(f\"\\nTable {i+1} from {pdf_file.name}:\")\n",
    "            display(df)\n",
    "\n",
    "            # Save individual table to CSV\n",
    "            csv_path = pdf_output_dir / f\"{pdf_file.stem}_table_{i+1}.csv\"\n",
    "            df.to_csv(csv_path, index=False)\n",
    "            print(f\"Saved to {csv_path}\")\n",
    "\n",
    "            # Log parsing report for each table\n",
    "            print(f\"\\nTable {i+1} Parsing Report:\")\n",
    "            logging.info(f\"Table {i+1} Parsing Report:\")\n",
    "            print(table.parsing_report)\n",
    "            logging.info(table.parsing_report)\n",
    "        except Exception as e:\n",
    "            print(f\"Failed to process or save table {i+1} from {pdf_file.name}: {e}\")\n",
    "            logging.error(f\"Failed to process or save table {i+1} from {pdf_file.name}: {e}\")\n",
    "\n",
    "# Define input_dir and output_dir\n",
    "input_dir = Path('/content/sample_pdfs')\n",
    "output_dir = Path('/content/output')\n",
    "\n",
    "print(f\"Input directory: {input_dir}\")\n",
    "print(f\"Output directory: {output_dir}\")\n",
    "\n",
    "# Ensure output directory exists\n",
    "output_dir.mkdir(exist_ok=True)\n",
    "\n",
    "# Process each PDF in the input directory\n",
    "pdf_files = list(input_dir.glob('*.pdf'))\n",
    "print(f\"Found {len(pdf_files)} PDF files\")\n",
    "\n",
    "if len(pdf_files) == 0:\n",
    "    print(\"No PDF files found in the input directory.\")\n",
    "    logging.warning(\"No PDF files found in the input directory.\")\n",
    "else:\n",
    "    for pdf_file in pdf_files:\n",
    "        process_pdf(pdf_file, output_dir)\n",
    "\n",
    "    print(\"Processing complete. Check the 'output' folder for results.\")\n",
    "    logging.info(\"Processing complete. Check the 'output' folder for results.\")\n",
    "\n",
    "print(\"Script execution finished.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "rCkxLY93k98_"
   },
   "outputs": [],
   "source": [
    "# @title ⚙️ Core - Complex Tables (Loose Parameters) with Clean Output\n",
    "\n",
    "# import pypdf_table_extraction\n",
    "# import os\n",
    "# from pathlib import Path\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tabulate import tabulate\n",
    "\n",
    "# Create output directory if it doesn't exist\n",
    "output_dir = Path('/content/output')\n",
    "output_dir.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "# Process all PDF files in the input directory\n",
    "input_dir = Path('/content/sample_pdfs')\n",
    "for pdf_file in input_dir.glob('*.pdf'):\n",
    "    print(f\"\\nProcessing {pdf_file.name}\")\n",
    "\n",
    "    # Using 'network' parsing method with table_areas\n",
    "    tables_network = pypdf_table_extraction.read_pdf(str(pdf_file), flavor='network')\n",
    "\n",
    "    if len(tables_network) == 0:\n",
    "        # If no tables are detected, try using 'lattice' parser\n",
    "        tables_lattice = pypdf_table_extraction.read_pdf(str(pdf_file), flavor='lattice', table_areas=['50,750,500,50'])\n",
    "\n",
    "    # Checking the detected tables\n",
    "    if len(tables_network) > 0:\n",
    "        tables = tables_network\n",
    "    elif len(tables_lattice) > 0:\n",
    "        tables = tables_lattice\n",
    "    else:\n",
    "        tables = []\n",
    "\n",
    "    # Exporting if tables are found\n",
    "    if len(tables) > 0:\n",
    "        output_base = output_dir / pdf_file.stem\n",
    "        tables.export(f'{output_base}.csv', f='csv', compress=True)  # export all tables to CSV\n",
    "        tables[0].to_csv(f'{output_base}_first_table.csv')  # Save the first table to CSV\n",
    "        df = tables[0].df  # Get the first table as a pandas DataFrame\n",
    "        \n",
    "        print(f\"Tables found in {pdf_file.name}:\")\n",
    "        \n",
    "        # Clean up the DataFrame\n",
    "        df = df.applymap(lambda x: x.strip() if isinstance(x, str) else x)  # Remove leading/trailing whitespace\n",
    "        df = df.replace(['', 'nan', 'NaN', 'NULL'], np.nan).dropna(how='all')  # Remove empty rows\n",
    "        df = df.fillna('')  # Replace NaN with empty string for display\n",
    "        df = df.reset_index(drop=True)  # Reset index after dropping rows\n",
    "        \n",
    "        # Display the clean DataFrame\n",
    "        print(tabulate(df, headers='keys', tablefmt='pretty', showindex=False))\n",
    "        \n",
    "        print(f\"\\nShape of the DataFrame: {df.shape}\")\n",
    "        print(f\"\\nParsing report: {tables[0].parsing_report}\")\n",
    "    else:\n",
    "        print(f\"No tables found in {pdf_file.name}\")\n",
    "\n",
    "print(\"\\nProcessing complete. Check the output directory for results.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "form",
    "id": "M1Ostjs9A2nQ"
   },
   "outputs": [],
   "source": [
    "# @title 🗑️ Clear Input & Output Directory\n",
    "\n",
    "import shutil\n",
    "# from pathlib import Path\n",
    "# import os\n",
    "\n",
    "# Define the directories to be cleared\n",
    "directories_to_clear = ['/content/output', '/content/sample_pdfs']\n",
    "\n",
    "# Warning message\n",
    "print(\"⚠️ WARNING: This will delete all contents of the following directories:\")\n",
    "for directory in directories_to_clear:\n",
    "    print(f\"- {directory}\")\n",
    "\n",
    "confirmation = input(\"Type 'YES' to confirm: \")\n",
    "\n",
    "if confirmation == 'YES':\n",
    "    for directory in directories_to_clear:\n",
    "        dir_path = Path(directory)\n",
    "        if dir_path.exists() and dir_path.is_dir():\n",
    "            # Remove all contents of the directory\n",
    "            for item in dir_path.iterdir():\n",
    "                if item.is_dir():\n",
    "                    shutil.rmtree(item)\n",
    "                else:\n",
    "                    item.unlink()\n",
    "            print(f\"✅ All contents of '{directory}' have been deleted.\")\n",
    "        else:\n",
    "            print(f\"The '{directory}' directory does not exist.\")\n",
    "else:\n",
    "    print(\"Operation cancelled. No files were deleted.\")\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "include_colab_link": true,
   "private_outputs": true,
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
